#!/usr/bin/python
# encoding: utf-8
import requests
import sys
from lxml import etree

reload(sys)
sys.setdefaultencoding('utf-8')

url='http://php20.cn/index/1'

def getNewUrlList():
    global url
    header ={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36'}    
    response=requests.get(url,headers=header)  
    html=response.content.decode("utf8")    
    selector=etree.HTML(html)
    contents = selector.xpath('//div[@class="row"]/div[@class="col-xs-12 col-md-12 col-lg-8"][div]')    
    # print(contents)
    for eachlink in contents:
    #     # url = eachlink.xpath('div/a/@href')[0] if str(eachlink.xpath('div/a/@href')[0]).__contains__("http") else "http://www.chinanews.com"+eachlink.xpath('div/a/@href')[0]
        title = eachlink.xpath('div/h3/a/text()')[0]

        print(title)
        print("-------------")
    #     # url=eachlink.xpath('div/a')[0]
    #     # ptime = eachlink.xpath('div[@class="dd_time"]/text()')[0]
    #     # yield (title,url,ptime)
    #     print(title)
        # yield (title)

        

def getNewContent(urlList):
    for title,url,ptime in urlList:
        response=requests.get(url,headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36'})
        html=response.content.decode("gbk")
        selector=etree.HTML(html)
        title=selector.xpath("//div[@id='cont_1_1_2']/h1/text()")[0]
        
        source=selector.xpath("//div[@id='cont_1_1_2']/div[@class='left-time']/div[@class='left-t']/text()")[0]
        content=selector.xpath("//div[@id='cont_1_1_2']/div[@class='left_zw']/p/text()")

        i=0
        resultContent=''
        for item in range(0,content.__len__()):
            resultContent+=content[i]
            i+=1
        yield (title,source,resultContent)

if __name__=="__main__":
   urlList= getNewUrlList()
#    print(urlList)

#    for item in urlList:
#        print(item)

#    result= getNewContent(urlList)
#    for title,source,content in result:
#        print u"标题:%s"%title
#        print u"来源：%s"%source
#        print u"正文:%s"%content
